#!/bin/bash

# Only ESMF needs to be built with the OpenMP flags now
profile_id_ESMF=ifort21_nc72_hybrid_ompi404
profile_id=ifort21_nc72_ompi404

# Add cmake
export PATH=/home/seastham/opt/cmake_3.16.5/bin:$PATH

IFORTVARS=/opt/intel/oneapi/setvars.sh
. $IFORTVARS

# Using Intel v21 compilers
export CC=icc
export CXX=icpc
export FC=ifort
export CPP=cpp

# Don't use F77/F90 for now
unset F77
unset F90
unset F9X

# Using /home/seastham/opt as a safe place for new packages
export opt_dir=/home/seastham/opt
export install_dir=${opt_dir}/pkg_${profile_id}
if [[ ! -d $install_dir ]]; then
   mkdir $install_dir
fi

export NETCDF_HOME=${install_dir}/netcdf
export NETCDF_FORTRAN_HOME=$NETCDF_HOME
export MPI_ROOT=${install_dir}/openmpi

# UCX
export PATH=/home/seastham/opt/pkg_${profile_id}/ucx/bin:$PATH
export LD_LIBRARY_PATH=/home/seastham/opt/pkg_${profile_id}/ucx/lib:$LD_LIBRARY_PATH

# MPI
export PATH=${MPI_ROOT}/bin:$PATH
export LD_LIBRARY_PATH=${MPI_ROOT}/lib:$LD_LIBRARY_PATH

# NetCDF
export PATH=${NETCDF_HOME}/bin:$PATH
export LD_LIBRARY_PATH=${NETCDF_HOME}/lib:$LD_LIBRARY_PATH

# Set compilers to MPI versions if available
if [[ $# -ge 1 && "$1" == "serial" ]]; then
   echo "Forcing serial compilers"
elif [[ -e $MPI_ROOT/bin/mpicc ]]; then
   echo "Setting compilers to MPI versions"
   export CC=mpicc
   export CXX=mpicxx
   export FC=mpif90
else
   echo "MPI compilers not found. Reverting to serial"
fi

# ESMF
export ESMF_DIR=/net/d14/data/seastham/ESMF_v800
export ESMF_INSTALL_PREFIX=${ESMF_DIR}/INSTALL_${profile_id_ESMF}
export ESMF_ROOT=$ESMF_INSTALL_PREFIX
export ESMF_COMPILER=intel
export ESMF_COMM=openmpi
export ESMF_OPENMP=ON

export NETCDF_C_ROOT=$NETCDF_HOME
export NETCDF_Fortran_ROOT=$NETCDF_HOME

# OpenMP setup
ulimit -s unlimited
ulimit -c 5000
export KMP_STACKSIZE=100000000

if [[ -z $SLURM_CPUS_PER_TASK ]]; then
   export OMP_NUM_THREADS=1
else
   export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
fi
export KMP_NUM_THREADS=$OMP_NUM_THREADS
